package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.task.CRIOnlineForSiChuanTask;
import com.spider.silence.task.ChinaNewsForSiChuanTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.MD5Utils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Created by admin on 2017/6/29.
 * 国际在线四川频道下载器
 */
public class CRIOnlineForSiChuanDownloader extends AbstractCommonDownLoader implements Cloneable {

    private Logger logger =Logger.getLogger(CRIOnlineForSiChuanDownloader.class);

    private Map<String,File> files = null;

    public Map<String, File> getFiles() {
        return files;
    }

    private final String crionline_site = "http://sc.cri.cn";

    //不同版块
    private final String[] section = {
            "http://sc.cri.cn/highlights.html", //要闻
            "http://sc.cri.cn/shizhou.html", //市州
            "http://sc.cri.cn/special.html", //专题
            "http://sc.cri.cn/feature.html", //原创
            "http://sc.cri.cn/plan.html", //策划
            "http://sc.cri.cn/entertainment.html", //体娱
            "http://sc.cri.cn/car.html", //汽车
            "http://sc.cri.cn/interview.html", //访谈
            "http://sc.cri.cn/company.html", //川企
            "http://sc.cri.cn/column.html", //专栏
            "http://sc.cri.cn/sichuan.html", //图片
            "http://sc.cri.cn/chengdu.html", //成都
            "http://sc.cri.cn/society.html", //社会
            "http://sc.cri.cn/finance.html", //财经
            "http://sc.cri.cn/video.html", //网视
            "http://sc.cri.cn/public.html", //舆情
            "http://sc.cri.cn/research.html", //调查
            "http://sc.cri.cn/education.html", //教育
            "http://sc.cri.cn/living.html", //房产
            "http://sc.cri.cn/travel.html", //乐游
            "http://sc.cri.cn/comments.html", //锐评
            "http://sc.cri.cn/food.html", //食品
            "http://sc.cri.cn/ecology.html", //环保
            "http://sc.cri.cn/voice.html" //民声
    };


    //生成爬虫任务
    public void generateTask(){
        for(int i = 0; i < section.length; i++) {
            //如果有异常重复十次确保生产任务完成
            for(int p = 0; p < 10; p++) {
                try {
                    Page page = getHtml(section[i],true);
                    if(page != null && page.getHtml() != null) {
                        Document document = page.getHtml().getDocument();
                        Elements elements = document.select("div[class=\"w1200_02\"] > div[class=\"content-box\"] > div[class=\"left\"] > div[class=\"left-ul list-box \"] >ul > li");
                        if(elements.size() > 0) {
                            for(int j = 0; j < elements.size(); j++) {
                                String url = crionline_site + elements.get(j).select("a").attr("href");
                                CRIOnlineForSiChuanTask task = new CRIOnlineForSiChuanTask(url);
                                CommonTaskManager.add("CRIOnlineForSiChuan", task);
                            }
                        }
                        break;
                    }
                } catch (Exception e) {
                    try {
                        Thread.sleep(3000);
                    } catch (InterruptedException e1) {
                        e1.printStackTrace();
                    }
                    e.printStackTrace();
                    continue;
                }
            }
        }
    }


    public JSONObject crawlItem(String url) throws Exception {
        JSONObject item = new JSONObject();
        JSONObject data;
        boolean success = false;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("CRIOnlineForSiChuan.timeOut"))) {
            data = new JSONObject();
            data.put("url", url);
            nowTime = System.currentTimeMillis();
            try {
                Page page = getHtml(url,true);
                if(page != null && page.getStatusCode() == 200) {
                    Document document = page.getHtml().getDocument();
                    String title = document.title();
                    data.put("title", title);

                    String publish_time = document.select("div[class=\"w1200_02\"] div[class=\"content-box\"] > div[class=\"left\"] > div[class=\"left-content\"] > div[class=\"left-content-title\"] > div[class=\"sign\"] > span").get(0).text();
                    publish_time = DateUtils.strToFormatStr(publish_time);
                    data.put("publish_time", publish_time); //新闻发布时间
                    data.put("spider_time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));//爬取时间
                    Elements content_img_p = document.select("div[id =\"abody\"] > p");
                    List<String> img_urls = new ArrayList<>();
                    StringBuilder content = new StringBuilder();
                    for(int i = 0; i < content_img_p.size(); i++) {
                        Element e = content_img_p.get(i);
                        if(e.select("img").size() > 0) {
                            img_urls.add(e.select("img").get(0).attr("src"));
                        } else {
                            content.append(e.text());
                        }
                    }
                    data.put("img_urls", img_urls.toArray());
                    data.put("content", content);
                    String newid = url.substring(url.lastIndexOf("/")+1, url.length()-5);
                    data.put("id", newid);
                    //设置html文件的命名格式
                    String fileName = "CRIOnlineForSiChuan" + "_" + DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime() + "_" + MD5Utils.getHash3(url.replaceAll("#.*", ""), "MD5");
                    String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                    String all_fileName = "国际在线四川频道_speeches_" + publishDate + "_" + fileName;
                    //抓取第一页的网页源码
                    File file = FileUtils.createTempFile(all_fileName + "_pageNo_1", document.html().toString());
                    files = new ConcurrentHashMap<>();
                    files.put(all_fileName + "_pageNo_1", file);
                    item.put("data", data);
                    success = true;
                    break;
                }
            }catch (Exception e) {
                Thread.sleep(1500);
                logger.info(e);
                System.err.println("爬虫过程出现异常, 尝试重新爬取");
                continue;
            }
        }
        if(success == true) {
            return item;
        } else {
            throw new Exception(url+"爬取失败");
        }
    }


}
